package com.sentiment.crawler;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.log4j.PropertyConfigurator;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sentiment.config.Config;
import com.sentiment.config.ConfigCrawler;
import com.sentiment.config.ConfigDb;
import com.sentiment.database.bean.CrawlNews;
import com.sentiment.database.bean.ForumInfo;
import com.sentiment.database.codec.CrawlNewsCodec;
import com.sentiment.database.codec.ForumInfoCodec;
import com.sentiment.database.dbUtil.DbCodecDriver;
import com.sentiment.nlpalgo.bean.Double2;
import com.sentiment.nlpalgo.bean.Keywords;
import com.sentiment.nlpalgo.method.AlgoHttp;
import com.sentiment.tools.Generator;
import com.sentiment.webcollector.crawler.DeepCrawler;
import com.sentiment.webcollector.model.Links;
import com.sentiment.webcollector.model.Page;

public class ForumInfoCrawler extends DeepCrawler {

	private static final Logger LOG = LoggerFactory.getLogger(ForumInfoCrawler.class);

	private volatile List<ForumInfo> infoList = new ArrayList<ForumInfo>();

	static {
		PropertyConfigurator.configure(Config.log4jPropertiesPath);
	}
	
	public ForumInfoCrawler(String crawlPath) {
		super(crawlPath);
	}
	
	/*
	 * 根据关键字 爬取论坛信息
	 */
	public void getSearch(String keyword) {
		try {
			addSeed(ConfigCrawler.tianyaCrw + URLEncoder.encode(keyword, "UTF-8"));
			start(2);
		} catch (Exception e) {
			LOG.error(e.toString());
		}
	}
	
	
	
	@Override
	public Links visitAndGetNextLinks(Page page) {
		String url = page.getUrl();
		Links links = new Links();
		if (Pattern.matches("http://search.tianya.cn/bbs\\?.*q=.*", url)) {
			links.addAllFromDocument(page.getDoc(), "h3>a");
			return links;
		} else {
			getForumInfo(page);
			return null;

		}
			
			
	}
	
	
	@SuppressWarnings({ "unchecked", "unused", "rawtypes" })
	public ForumInfo getForumInfo(Page page){
		ForumInfo foruminfo = new ForumInfo();
		Date date = new Date();
		foruminfo.setDate(date);
		Document doc = Jsoup.parse(page.getHtml());
		Elements es = doc.getElementsByAttributeValue("class", "crumbs");
		Elements ess = doc.getElementsByAttributeValue("class", "bbs-content clearfix");
		ArrayList<String> texts = new ArrayList();
		if(ess.text()!=null&&ess.text()!=""){
			texts.add(ess.text());
		}
		
		if (Config.useNlp) {
			AlgoHttp algoHttp = new AlgoHttp();

			foruminfo.setSentiment(algoHttp.sentiment(texts, AlgoHttp.SENTIMENT_NEWS));

			foruminfo.setKeywords(algoHttp.keywords(texts));
		} else {
			// 随机生成的测试数据 ， 只生成3组测试数据
			List<Double2> list = new ArrayList<Double2>();
			for (int i = 0; i < 3; i++) {
				double rd = Generator.genenrateDouble(0, 1);
				list.add(new Double2(1 - rd, rd));
			}
			foruminfo.setSentiment(list);
			List<Keywords> list2 = new ArrayList<Keywords>();

			for (int i = 0; i < 3; i++) {
				String keyword = Generator.generateKeyword();
				list2.add(new Keywords(Generator.genenrateDouble(0, 1), keyword));
			}
			foruminfo.setKeywords(list2);
		}
		foruminfo.setText(texts);
		String from = es.select("em").text();
		foruminfo.setFrom(from);
		Element element =  doc.getElementById("post_head");
		Elements title = element.select("h1");
		Elements info = element.select("span");
		foruminfo.setUrl(page.getUrl());
		String infos = info.text();
		
		foruminfo.setTitle(title.first().text());
		String publisher = info.get(4).text().split("：")[1];
		foruminfo.setPublisher(publisher);
		String rtime = info.get(5).text().split("：")[1];
		Date retime = getpubtime(rtime);
		foruminfo.setReleaseTime(retime);
		
		int clicktime = Integer.parseInt(info.get(6).text().split("：")[1]);
		System.out.print(clicktime+"  ");
		int total = Integer.parseInt(info.get(7).text().split("：")[1]);
		foruminfo.setClicktimes(clicktime);
		foruminfo.setComment(total);
		ArrayList<String> arr = new ArrayList();
		for(int i = 1 ;i<=total&&i<=100 ;i++){
			try{
				Element eleme = doc.getElementById(i+"");
				if(eleme!=null){
					String str = eleme.getElementsByAttributeValue("class", "bbs-content").text().replace(" ", "").replace("\n\r", "");
					
					if(str!=null&&str!=""){
						arr.add(str);
					}
				}
				
			}catch(Exception w){
				w.printStackTrace();
			}
			
		}
		try{
//			if(arr.isEmpty()){
//				arr.add("无回复");
//			}
			foruminfo.setComTexts(arr);
		}catch(Exception e){
			e.printStackTrace();
		}
		infoList.add(foruminfo);
		return foruminfo;
	}
	
	@SuppressWarnings("deprecation")
	public Date getpubtime(String rtime){
		Date date = new Date();
		String[] ss = rtime.split(" ");
		String[] ss1 = ss[0].split("-");
		String[] ss2 = ss[1].split(":");
		try{
			date.setYear(Integer.parseInt(ss1[0])-1900);
			date.setMonth(Integer.parseInt(ss1[1])-1);
			date.setDate(Integer.parseInt(ss1[2]));
			date.setHours(Integer.parseInt(ss2[0]));
			date.setMinutes(Integer.parseInt(ss2[1]));
			date.setSeconds(Integer.parseInt(ss2[2]));

		}catch(Exception e){
			e.printStackTrace();
		}
		
		
		return date;
	}
	
	
	public List<ForumInfo> getInfoList() {
		return infoList;
	}

	public void setInfoList(List<ForumInfo> infoList) {
		this.infoList = infoList;
	}

	/*
	 * 将爬虫爬取到的数据插入 ForumInfo表中
	 */
	@SuppressWarnings("rawtypes")
	public void insertIntoDB(){
		Iterator it = infoList.iterator();
		DbCodecDriver<ForumInfo> driver = new DbCodecDriver<ForumInfo>();
		driver.openConnection(ConfigDb.sentimentDb, ForumInfo.class, ForumInfoCodec.class);
		System.out.println("length:"+infoList.size());
		try{
			while(it.hasNext()){
				driver.insert((ForumInfo) it.next());
				
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		
		driver.closeConnection();

	}
	
	
	@SuppressWarnings("deprecation")
	public static void main(String[] args) {
		ForumInfoCrawler crawler = new ForumInfoCrawler(Config.crawlerDataPath);
		try{
			crawler.getSearch("北京");

			
			crawler.insertIntoDB();
		}catch(Exception e){
			e.printStackTrace();
		}


	}
}
